package Spark招聘数据处理

import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @author 35192
 * @date 2021-05-22 17:47
 */
object Task01 {

  // 创建SparkContext
  val conf = new SparkConf()
    .setAppName("log_analysis")
    .setMaster("local[6]")
  val sc = new SparkContext(conf)
  // 设置日志级别
  sc.setLogLevel("ERROR")
  // 创建SparkSession
  val spark = SparkSession.builder()
    .master("local[6]")
    .appName("log")
    .getOrCreate()
  // 导入隐式转换和函数集
  import org.apache.spark.sql.functions._
  import spark.implicits._

  /**
   * TODO 薪资处理
   * @param salary
   * @return
   */
  def parseSalary(salary: String) = {
    // 3.1 面议处理
    var all = "";
    var face = "";
    var min_salary = 0;
    var max_salary = 0;
    if (salary.startsWith("(面议)")) {
      face = "是"
      // 测试工程师,(面议)5-8K,中山市沐捷电器科技有限公司,广东/中山市/阜沙镇,1-2年,大专
      all = salary.substring(salary.indexOf(')') + 1, salary.length()) // 5-8K
      // 判空
      if (null != all && !"".equals(all)) {
        //  3.2 提取最大最小值
        val salaries = all.split("-")
        min_salary = salaries(0).toInt
        max_salary = salaries(1).substring(0, salaries(1).length - 1).toInt
      } else {
        min_salary = 0 ;
        max_salary = 0;
      }
    } else {
      face = "否"
      // 软件工程师,5-8K,中山市利和物业管理有限公司,广东/中山市/东区,1-2年,大专
      if (null != salary && !"".equals(salary)) {
        //  3.2 提取最大最小值
        all = salary
        val salaries = all.split("-")
        min_salary = salaries(0).toInt
        max_salary = salaries(1).substring(0, salaries(1).length - 1).toInt
      } else {
        min_salary = 0;
        max_salary = 0;
      }
    }
    (all,face,min_salary,max_salary)
  }

  def main(args: Array[String]): Unit = {

    // 读取数据
    val data = sc.textFile("G:\\Projects\\IdeaProjects\\Spark_Competition\\dataset\\java_rencai.csv")

    val areas = Set[String]()

    val clean = data.map {
      item =>
        // 1.去引号，拆分
        val line = item.replace("\"", "").split(",")
        // 2.获取字段值
        val position = line(0)
        val salary = line(1)
        val company = line(2)
        val area = line(3)
        val experiences = line(4)
        val education = line(5)

        // 3.薪资处理
        val sal = parseSalary(salary)._1;
        val face = parseSalary(salary)._2
        val min = parseSalary(salary)._3
        val max = parseSalary(salary)._4
        (position,sal,face,min,max)


    }
    clean.foreach(println(_))

  }
}
